﻿#!/usr/bin/env perl
=pod

=head1 NOTE

How to create the data file for this script:
1. fetch unihan.txt from unicode.org
2. grep kFrequency Unihan.txt > kfreq.txt	// in my own version, it is 5089 lines

=cut
use strict;
use Storable;
use Getopt::Std;

my $store_file = 'kfreq.dat';
my $freq_file = 'kfreq.txt';
my %freq = ();

sub load_freq_file($)
{
	my $file = shift;
	my $cnt = 0;

	print STDERR "load frequency data from text format: $file ...\n";
	open my $fh, $file or die;
	while (<$fh>)  {
		next if (m/^$/ || m/^#/);
		m/^U\+([0-9a-fA-F]+)/;
		my $ucp = $1 if $1;
		#print $ucp,"\t";
		#binmode(STDOUT, ':encoding(utf8)');
		#print chr(hex($ucp)),"\n";
		$freq{ chr(hex($ucp)) } = 1;
		$cnt ++;
	}
	close $fh;
	print STDERR "items loaded from text file: $cnt\n";
	store(\%freq, $store_file);
}

sub filter_lime_table($)
{
	my $file = shift;
	my $ofile = $file;
	my ($matched, $deleted, $symbol) = (0, 0, 0);
	my ($iline, $oline) = (0, 0);
	my %ih = ();
	my %oh = ();

	# make output file name
	$ofile =~ m/(\w+)(\.\w+)/;
	$ofile = sprintf("%s-mini%s", $1, $2);

	#binmode(STDOUT, ':encoding(utf8)');
	open my $fh, $file or die;
	binmode($fh, ':encoding(utf8)');
	open my $ofh, "> $ofile" or die;
	binmode($ofh, ':encoding(utf8)');

	open my $mfh, "> sym.txt" or die;
	binmode($mfh, ':encoding(utf8)');

	while ( <$fh> )  {
		$iline ++;
		my $ln = $_;
		m/^(\S+)\t(\S+)$/;

		next unless $1;
		next unless $2;
		my ($kd, $vl) = ($1, $2);
		$ih{$vl} = 1;	# to see input unique lines

		if ($freq{$vl})  {	# found in %freq
			$matched ++;
			$oline ++;
			$oh{$vl} = 1;	# to see output unique characters
			print $ofh $ln;
		}
		if ($kd =~ m/[,:\@\#\$\%\!\/\.\[\]\']/)  {
#			$symbol ++;
#			$oline ++;
#			$oh{$vl} = 1;	# to see output unique symbols
#			print $ofh $ln;
			print $mfh $ln;
		}
		else  {
			$deleted ++;
		}
	}
	close $fh;
	close $ofh;
	close $mfh;

	printf STDERR "output to: $ofile\n";
	printf STDERR "kept:%d, deleted:%d, symbol(also kept):%d\n", $matched, $deleted, $symbol;
	printf STDERR "input def: %d, output def: %d\n", $iline, $oline;
	printf STDERR "unique input: %d, output: %d\n", scalar(keys(%ih)), scalar(keys(%oh));
}

sub see_lime($)
{
	my $file = shift;
	my $total_kd = 0;	# total key define
	my %sym = ();	# total symbols in table

	open my $fh, $file or die;
	binmode($fh, ':encoding(utf8)');
	while ( <$fh> )  {
		next if m/^$/;
		$total_kd ++;
		m/^(\S+)\t(\S+)$/;
		if ($2)  {
			$sym{$2} = 1;
		}
	}
	close $fh;
	printf "total key define: %d, unique symbols: %d",
		$total_kd, scalar(keys(%sym));
}


# input:
# 1. a reference to an array
# 2. output file name
sub output($$)
{
	my $rf = shift;
	my $ofile = shift;
	open my $ofh, "> $ofile" or die;
	print "output to: $ofile\n";
	binmode($ofh, ':encoding(utf8)');
	print $ofh "# this file is outputed by ''mkfreq.pl -s''\n";
	print $ofh "# first colume is unicode code point U+[0-9a-fA-F]+\n";

	my @arr = @$rf;
	foreach (sort @arr)  {
		printf $ofh "U+%X\t%s\n", ord($_), $_;
	}

	close $ofh;
}

sub main
{
	unless (@ARGV)  {
		print STDERR "please specify the lime file...\n";
		print STDERR "perl mkfreq.pl [-f] <foo.lime>\n";
		print STDERR "-c limefile	show some info about lime file\n";
		print STDERR "-f	build kfreq.dat and exit\n";
		print STDERR "-s	show kfreq.dat to kf-show.txt and exit\n";
		return;
	}

	my %opts = ();
	getopts('c:fs', \%opts);

	# -c limefile	show some info about lime file
	if ( $opts{'c'} )  {
		see_lime($opts{'c'});
		return;
	}
	# -f	build kfreq.dat and exit
	if ( $opts{'f'} )  {
		load_freq_file($freq_file);
		return;
	}
	# load data from storage file
	if (-e $store_file)  {
		print STDERR "load $store_file ...\n";
		%freq = %{ retrieve($store_file) };   # direct to hash
	} else  {
		load_freq_file($freq_file);
	}
	# -s	show kfreq.dat to kf-show.txt and exit
	if ( $opts{'s'} )  {
		my @fr = keys(%freq);
		output(\@fr, "kf-show.txt");
		return;
	}

	print STDERR "size of table: ", scalar( keys(%freq) ), "\n";
	filter_lime_table($ARGV[0]);
}

main;
